STEP 1 - Clean and identify mistakes from pq_parser script

This notebook: 1. Reads in the output from pq_parser.ipynb (“pq_metadata.csv”) 2. Cleans the output from pq_parser.ipynb 3. Writes cleaned corpus (“01_pq_metaclean.csv”)

load in data

#load data
pq_metadata <- data.table::fread('Data/02_Working/pq_metadata.csv')

# displays column names & number of rows - 6589
names(pq_metadata);nrow(pq_metadata)
 [1] "Title"                   "Publication title"       "Publication year"        "Document URL"            "Full text"              
 [6] "Links"                   "Section"                 "Publication subject"     "ISSN"                    "Copyright"              
[11] "Abstract"                "Publication info"        "Last updated"            "Place of publication"    "Location"               
[16] "Author"                  "Publisher"               "Identifier / keyword"    "Source type"             "ProQuest document ID"   
[21] "Country of publication"  "Language of publication" "Publication date"        "Subject"                 "Database"               
[26] "Document type"          
[1] 6589
# Head displays the first 6 rows of the data.table
head(pq_metadata)

check empty full texts

# How do you want to treat NA?
# subset the data by the NAs, explore them & their origin
# can we programmatically fix it, or do we have to do it manually?

# 51 rows blank/NA Full text
pq_empty<-pq_metadata[pq_metadata$`Full text` == "" | is.na(pq_metadata$`Full text`),]
nrow(pq_empty)
[1] 51

remove empty full texts & missing IDs

# How do you want to treat NA?
# subset the data by the NAs, explore them & their origin
# can we programmatically fix it, or do we have to do it manually?

# 50 blank/NA pub titles
#pq_metaclean<-pq_metadata[!(pq_metadata$`Full text` == "" | is.na(pq_metadata$`Full text`)),]

pq_metaclean <- pq_metadata %>% 
  drop_na(`Full text`) %>%
  drop_na(`ProQuest document ID`) %>%
  filter(`Full text` != "") %>%
  filter(`ProQuest document ID` != "")

numReduced<- nrow(pq_metadata)-nrow(pq_metaclean)
print(paste("NA or blank Full Text rows removed:",numReduced))
[1] "NA or blank Full Text rows removed: 128"

check unique publication titles

# what are the unique publication titles 
unique(pq_metaclean$`Publication title`)
 [1] "Bangor Daily News; Bangor, Me."        "Bangor Daily News; Bango r, Me."       "Bangor Daily News; Bang or, Me."      
 [4] "Bang or Daily News; Bangor, Me."       "Kennebec Journal; Augusta, Me."        "Maine Times; Portland, Me."           
 [7] "Morning Sentinel; Waterville, Me."     "Portland Press Herald; Portland, Me."  "Portland Press Herald; Port land, Me."
[10] "Portland Pr ess Herald; Portland, Me." "Portland Pre ss Herald; Portland, Me." "Portl and Press Herald; Portland, Me."
[13] "Portland Press Herald Portland, Me."   "Portlan d Press Herald; Portland, Me." "Port land Press Herald; Portland, Me."
[16] ""                                      "Sun Journal; Lewiston, Me."           

check empty publication titles

# How do you want to treat NA?
# subset the data by the NAs, explore them & their origin
# can we programmatically fix it, or do we have to do it manually?

# 15 blank/NA pub titles (there were 50 before Full text clean)
pq_empty<-pq_metaclean[pq_metaclean$`Publication title` == "" | is.na(pq_metaclean$`Publication title`),]
nrow(pq_empty)
[1] 1
# 0 of these blank/NA pub titles have blank/NA 'full texts' as well (there were 37 before full text clean)
nrow(pq_empty[pq_empty$`Full text` == "" | is.na(pq_empty$`Full text`),])
[1] 0

if publication title is blank or NA, replace it with the publication title in publication info

pq_metaclean <- pq_metaclean %>% 
  mutate(`Publication title` = ifelse(`Publication title` == "" | is.na(`Publication title`), 
                                      unlist(strsplit(`Publication info`, split="\\[|:"))[1], `Publication title`))

#check to see if pub title substitution worked 
unique(pq_metaclean$`Publication title`)
 [1] "Bangor Daily News; Bangor, Me."        "Bangor Daily News; Bango r, Me."       "Bangor Daily News; Bang or, Me."      
 [4] "Bang or Daily News; Bangor, Me."       "Kennebec Journal; Augusta, Me."        "Maine Times; Portland, Me."           
 [7] "Morning Sentinel; Waterville, Me."     "Portland Press Herald; Portland, Me."  "Portland Press Herald; Port land, Me."
[10] "Portland Pr ess Herald; Portland, Me." "Portland Pre ss Herald; Portland, Me." "Portl and Press Herald; Portland, Me."
[13] "Portland Press Herald Portland, Me."   "Portlan d Press Herald; Portland, Me." "Port land Press Herald; Portland, Me."
[16] "Bangor Daily News ; Bangor, Me. "      "Sun Journal; Lewiston, Me."           

fix duplicate publication titles

# Substitute 
# "Bangor Daily News; Bang or, Me."  
# "Bangor Daily News; Ba ngor, Me." 
# "Bangor Dail y News; Bangor, Me."  
# with "Bangor Daily News; Bangor, Me."  

# "Morning Sentinel; Wate rville, Me." 
# "Central Maine Morning Sentinel; Waterville, Me."
#  with "Morning Sentinel; Waterville, Me."       

# "Kennebec Journal; Augusta, Me." 
# "Kennebec Journal; Augusta, Me ."

pq_metaclean <- pq_metaclean %>% 
  mutate(`Publication title` = recode(`Publication title`,
                                      'Bangor Daily News; Bang or, Me.' = 'Bangor Daily News; Bangor, Me.',
                                      'Bangor Daily News; Ba ngor, Me.' = 'Bangor Daily News; Bangor, Me.',
                                      'Bangor Dail y News; Bangor, Me.' = 'Bangor Daily News; Bangor, Me.',
                                      'Bangor Daily News; Bango r, Me.' = 'Bangor Daily News; Bangor, Me.',
                                      'Bang or Daily News; Bangor, Me.' = 'Bangor Daily News; Bangor, Me.',
                                      "Bangor Daily News ; Bangor, Me. " = 'Bangor Daily News; Bangor, Me.',
                                      'Morning Sentinel; Wate rville, Me.' = 'Morning Sentinel; Waterville, Me.',
                                      'Central Maine Morning Sentinel; Waterville, Me.' = 'Morning Sentinel; Waterville, Me.',
                                      'Kennebec Journal; Augusta, Me .' = 'Kennebec Journal; Augusta, Me.',
                                      'Portland Press Herald; Port land, Me.' = 'Portland Press Herald; Portland, Me.',
                                      'Portland Pr ess Herald; Portland, Me.' = 'Portland Press Herald; Portland, Me.',
                                      'Portland Pre ss Herald; Portland, Me.' = 'Portland Press Herald; Portland, Me.',
                                      'Portl and Press Herald; Portland, Me.' = 'Portland Press Herald; Portland, Me.',
                                      'Portland Press Herald Portland, Me.' = 'Portland Press Herald; Portland, Me.',
                                      'Portlan d Press Herald; Portland, Me.' = 'Portland Press Herald; Portland, Me.',
                                      'Port land Press Herald; Portland, Me.' = 'Portland Press Herald; Portland, Me.',
                                      'Portlan d Press Herald; Portland, Me.' = 'Portland Press Herald; Portland, Me.',
                                      'Port land Press Herald; Portland, Me.' = 'Portland Press Herald; Portland, Me.'))

#check to see if pub title substitution worked --- yes, no weird repeats
unique(pq_metaclean$`Publication title`)
[1] "Bangor Daily News; Bangor, Me."       "Kennebec Journal; Augusta, Me."       "Maine Times; Portland, Me."           "Morning Sentinel; Waterville, Me."   
[5] "Portland Press Herald; Portland, Me." "Sun Journal; Lewiston, Me."          

#check unique Publication years

unique(pq_metaclean$'Publication year')
 [1] "1997"  "1996"  "1995"  "1994"  "1993"  "2004"  "2003"  "200 3" "2002"  "2001"  "2009"  "2008"  "2007"  "2006"  "2005"  "2014"  "2013"  "2012"  "2011" 
[20] "2010"  "2019"  "2018"  "2017"  "2016"  "2015"  "2000"  "1999"  "1998"  "201 8" ""     

#clean publication years

pq_metaclean <- pq_metaclean %>%
  mutate(`Publication year` = recode(`Publication year`,
                                     '200 3' = '2003',
                                     '201 8' = '2018'))

#check to see if pub title substitution worked --- yes, no weird repeats
unique(pq_metaclean$`Publication year`)
 [1] "1997" "1996" "1995" "1994" "1993" "2004" "2003" "2002" "2001" "2009" "2008" "2007" "2006" "2005" "2014" "2013" "2012" "2011" "2010" "2019" "2018" "2017"
[23] "2016" "2015" "2000" "1999" "1998" ""    

check empty publication years

# How do you want to treat NA?
# subset the data by the NAs, explore them & their origin
# can we programmatically fix it, or do we have to do it manually?

# 3 blank/NA pub years
pq_empty<-pq_metaclean[pq_metaclean$`Publication year` == "" | is.na(pq_metaclean$`Publication year`),]
nrow(pq_empty)
[1] 3
# 0 of these blank/NA pub years have blank/NA 'full texts' as well
nrow(pq_empty[pq_empty$`Full text` == "" | is.na(pq_empty$`Full text`),])
[1] 0

if publication year is blank or NA, replace it with the year in publication info

#pq_metaclean <- pq_metaclean %>%
#  mutate(`Publication year` = ifelse(`Publication year` == "" | is.na(`Publication year`), str_sub(`Publication date`,-4,-1), `Publication year`))

# Sub from last 4 digits in publication date
# sub from matched condition of being 4 digits in publication info
pq_metaclean <- pq_metaclean %>%
  mutate(`Publication year` = ifelse(`Publication year` == "" | is.na(`Publication year`), 
                                     sub('.*(\\d{4}).*', '\\1', `Publication info`), `Publication year`))
#check to see if pub title substitution worked --- yes, no weird repeats
unique(pq_metaclean$`Publication year`)
 [1] "1997" "1996" "1995" "1994" "1993" "2004" "2003" "2002" "2001" "2009" "2008" "2007" "2006" "2005" "2014" "2013" "2012" "2011" "2010" "2019" "2018" "2017"
[23] "2016" "2015" "2000" "1999" "1998"

re-check empty publication years (should be 0 now)

# How do you want to treat NA?
# subset the data by the NAs, explore them & their origin
# can we programmatically fix it, or do we have to do it manually?

# 3 blank/NA pub years
pq_empty<-pq_metaclean[pq_metaclean$`Publication year` == "" | is.na(pq_metaclean$`Publication year`),]
nrow(pq_empty)
[1] 0
# 0 of these blank/NA pub years have blank/NA 'full texts' as well
nrow(pq_empty[pq_empty$`Full text` == "" | is.na(pq_empty$`Full text`),])
[1] 0

check empty publication dates

# How do you want to treat NA?
# subset the data by the NAs, explore them & their origin
# can we programmatically fix it, or do we have to do it manually?

# 3 blank/NA pub years
pq_empty<-pq_metaclean[pq_metaclean$`Publication date` == "" | is.na(pq_metaclean$`Publication date`),]
nrow(pq_empty)
[1] 2
# 0 of these blank/NA pub years have blank/NA 'full texts' as well
nrow(pq_empty[pq_empty$`Full text` == "" | is.na(pq_empty$`Full text`),])
[1] 0
# if publication date is empty or NA, fill with date from publication info
pq_metaclean <- pq_metaclean %>% 
  mutate(`Publication date` = ifelse(`Publication date` == "" | is.na(`Publication date`), 
                                     unlist(strsplit(`Publication info`, split="\\]|:"))[2], `Publication date`))

re-check empty publication dates (should be 0 now)

# How do you want to treat NA?
# subset the data by the NAs, explore them & their origin
# can we programmatically fix it, or do we have to do it manually?

# 3 blank/NA pub years
pq_empty<-pq_metaclean[pq_metaclean$`Publication date` == "" | is.na(pq_metaclean$`Publication date`),]
nrow(pq_empty)
[1] 0
# 0 of these blank/NA pub years have blank/NA 'full texts' as well
nrow(pq_empty[pq_empty$`Full text` == "" | is.na(pq_empty$`Full text`),])
[1] 0

more cleanings

PubTitles<-"Bangor Daily News|Kennebec Journal|Maine Times|Morning Sentinel|Portland Press Herald|Sun Journal" 
TxtFormatting<-"\a|\b|\f|\n|\r|\t|\v|\\[|\\]"

pq_metaclean<- pq_metaclean %>%
  mutate(`Full text` = textclean::replace_html(`Full text`, symbol=TRUE)) %>% # replace html, url, symbol, white from textclean
  mutate(`Full text` = textclean::replace_url(`Full text`, replacement = '<<URL>>')) %>%
  mutate(`Full text` = gsub(PubTitles, " ", `Full text`)) %>% # remove publication titles
  mutate(`Full text` = gsub(TxtFormatting, " ", `Full text`)) %>% # remove text formatting
  mutate(`Full text` = gsub("\"", " ", `Full text`)) %>% # remove double quotes ""
  mutate(`Full text` = stringi::stri_trans_general(str = `Full text`, id = "Latin-ASCII")) %>% # remove accented letters
  mutate(`Full text` = custom_replace_symbol(`Full text`)) %>% # replace "&" with "and" and "%" with "percent"
  mutate(`Full text` = textclean::replace_white(`Full text`)) # remove extra white space 

If duplicate full text exists, select first in series and dump the rest? – yes, since

initial cleaning is to produce representative categories

These document IDs have random spaces in the publication date; i.e., May 01, 20 14 DID1<-“1540791926” DID2<-“2257133415”

pq_metadata[pq_metadata$ProQuest document ID==DID1,]

#
head(pq_metaclean)

sort(pq_metaclean$`Publication date`)
   [1] "22 Nov 1997"  "22 Nov 1997"  "Apr 08, 1994" "Apr 08, 2011" "Apr 1, 1994"  "Apr 1, 1997"  "Apr 1, 1997"  "Apr 1, 1998"  "Apr 1, 2001"  "Apr 1, 2003" 
  [11] "Apr 1, 2005"  "Apr 1, 2006"  "Apr 1, 2007"  "Apr 1, 2007"  "Apr 1, 2010"  "Apr 1, 2012"  "Apr 1, 2014"  "Apr 1, 2014"  "Apr 1, 2014"  "Apr 1, 2014" 
  [21] "Apr 1, 2014"  "Apr 1, 2018"  "Apr 1, 2018"  "Apr 10, 1996" "Apr 10, 1996" "Apr 10, 1997" "Apr 10, 1998" "Apr 10, 2000" "Apr 10, 2001" "Apr 10, 2003"
  [31] "Apr 10, 2004" "Apr 10, 2007" "Apr 10, 2011" "Apr 10, 2012" "Apr 10, 2014" "Apr 10, 2015" "Apr 10, 2016" "Apr 10, 2019" "Apr 10, 2019" "Apr 11, 1994"
  [41] "Apr 11, 1996" "Apr 11, 1996" "Apr 11, 1998" "Apr 11, 2001" "Apr 11, 2002" "Apr 11, 2006" "Apr 11, 2007" "Apr 11, 2008" "Apr 11, 2013" "Apr 11, 2016"
  [51] "Apr 11, 2017" "Apr 11, 2018" "Apr 11, 2018" "Apr 11, 2018" "Apr 11, 2019" "Apr 12, 1994" "Apr 12, 1997" "Apr 12, 1997" "Apr 12, 1997" "Apr 12, 2002"
  [61] "Apr 12, 2004" "Apr 12, 2010" "Apr 12, 2012" "Apr 12, 2012" "Apr 12, 2012" "Apr 12, 2013" "Apr 12, 2016" "Apr 12, 2017" "Apr 12, 2017" "Apr 13, 1994"
  [71] "Apr 13, 1996" "Apr 13, 1996" "Apr 13, 2001" "Apr 13, 2004" "Apr 13, 2004" "Apr 13, 2005" "Apr 13, 2008" "Apr 13, 2008" "Apr 13, 2008" "Apr 13, 2013"
  [81] "Apr 13, 2014" "Apr 13, 2014" "Apr 13, 2015" "Apr 13, 2016" "Apr 13, 2016" "Apr 13, 2017" "Apr 13, 2019" "Apr 14, 1995" "Apr 14, 1996" "Apr 14, 1997"
  [91] "Apr 14, 1999" "Apr 14, 2000" "Apr 14, 2001" "Apr 14, 2002" "Apr 14, 2003" "Apr 14, 2003" "Apr 14, 2010" "Apr 14, 2010" "Apr 14, 2012" "Apr 14, 2016"
 [101] "Apr 14, 2019" "Apr 15, 1997" "Apr 15, 1997" "Apr 15, 1997" "Apr 15, 1998" "Apr 15, 1998" "Apr 15, 1998" "Apr 15, 2001" "Apr 15, 2003" "Apr 15, 2003"
 [111] "Apr 15, 2006" "Apr 15, 2015" "Apr 15, 2016" "Apr 15, 2016" "Apr 15, 2018" "Apr 16, 1997" "Apr 16, 1998" "Apr 16, 1999" "Apr 16, 2003" "Apr 16, 2005"
 [121] "Apr 16, 2007" "Apr 16, 2008" "Apr 16, 2008" "Apr 16, 2015" "Apr 16, 2016" "Apr 16, 2019" "Apr 16, 2019" "Apr 16, 2019" "Apr 17, 1996" "Apr 17, 1996"
 [131] "Apr 17, 1996" "Apr 17, 1998" "Apr 17, 2001" "Apr 17, 2001" "Apr 17, 2002" "Apr 17, 2004" "Apr 17, 2013" "Apr 17, 2014" "Apr 17, 2015" "Apr 17, 2016"
 [141] "Apr 17, 2017" "Apr 17, 2017" "Apr 17, 2017" "Apr 17, 2017" "Apr 17, 2018" "Apr 17, 2018" "Apr 17, 2018" "Apr 17, 2019" "Apr 17, 2019" "Apr 17, 2019"
 [151] "Apr 17, 2019" "Apr 17, 2019" "Apr 18, 1999" "Apr 18, 2001" "Apr 18, 2003" "Apr 18, 2007" "Apr 18, 2007" "Apr 18, 2007" "Apr 18, 2010" "Apr 18, 2013"
 [161] "Apr 18, 2014" "Apr 18, 2017" "Apr 18, 2018" "Apr 18, 2018" "Apr 18, 2019" "Apr 18, 2019" "Apr 19, 1994" "Apr 19, 1994" "Apr 19, 1996" "Apr 19, 1997"
 [171] "Apr 19, 1998" "Apr 19, 1998" "Apr 19, 1998" "Apr 19, 1998" "Apr 19, 1998" "Apr 19, 2000" "Apr 19, 2006" "Apr 19, 2007" "Apr 19, 2011" "Apr 19, 2012"
 [181] "Apr 19, 2013" "Apr 19, 2017" "Apr 2, 1997"  "Apr 2, 1999"  "Apr 2, 2000"  "Apr 2, 2003"  "Apr 2, 2003"  "Apr 2, 2007"  "Apr 2, 2009"  "Apr 2, 2009" 
 [191] "Apr 2, 2009"  "Apr 2, 2009"  "Apr 2, 2010"  "Apr 2, 2012"  "Apr 2, 2014"  "Apr 2, 2014"  "Apr 2, 2014"  "Apr 2, 2014"  "Apr 2, 2015"  "Apr 2, 2017" 
 [201] "Apr 2, 2019"  "Apr 20, 1997" "Apr 20, 1997" "Apr 20, 1997" "Apr 20, 1997" "Apr 20, 1999" "Apr 20, 2000" "Apr 20, 2000" "Apr 20, 2002" "Apr 20, 2004"
 [211] "Apr 20, 2005" "Apr 20, 2006" "Apr 20, 2006" "Apr 20, 2010" "Apr 20, 2011" "Apr 20, 2013" "Apr 20, 2014" "Apr 20, 2016" "Apr 20, 2019" "Apr 20, 2019"
 [221] "Apr 21, 1996" "Apr 21, 1998" "Apr 21, 1999" "Apr 21, 2000" "Apr 21, 2003" "Apr 21, 2007" "Apr 21, 2010" "Apr 21, 2010" "Apr 21, 2015" "Apr 21, 2017"
 [231] "Apr 21, 2019" "Apr 22, 1995" "Apr 22, 1996" "Apr 22, 1997" "Apr 22, 1998" "Apr 22, 1998" "Apr 22, 2000" "Apr 22, 2004" "Apr 22, 2004" "Apr 22, 2015"
 [241] "Apr 22, 2017" "Apr 22, 2017" "Apr 22, 2018" "Apr 22, 2018" "Apr 23, 1997" "Apr 23, 1997" "Apr 23, 1997" "Apr 23, 1999" "Apr 23, 2000" "Apr 23, 2000"
 [251] "Apr 23, 2000" "Apr 23, 2001" "Apr 23, 2004" "Apr 23, 2004" "Apr 23, 2005" "Apr 23, 2007" "Apr 23, 2009" "Apr 23, 2012" "Apr 23, 2013" "Apr 23, 2014"
 [261] "Apr 23, 2014" "Apr 23, 2016" "Apr 23, 2017" "Apr 23, 2017" "Apr 24, 1995" "Apr 24, 1996" "Apr 24, 1996" "Apr 24, 1997" "Apr 24, 1998" "Apr 24, 1999"
 [271] "Apr 24, 2000" "Apr 24, 2002" "Apr 24, 2004" "Apr 24, 2010" "Apr 24, 2012" "Apr 24, 2016" "Apr 24, 2016" "Apr 24, 2016" "Apr 24, 2016" "Apr 25, 1998"
 [281] "Apr 25, 1999" "Apr 25, 2000" "Apr 25, 2000" "Apr 25, 2000" "Apr 25, 2000" "Apr 25, 2000" "Apr 25, 2003" "Apr 25, 2007" "Apr 25, 2011" "Apr 25, 2016"
 [291] "Apr 25, 2016" "Apr 25, 2016" "Apr 25, 2018" "Apr 25, 2019" "Apr 25, 2019" "Apr 26, 1995" "Apr 26, 1998" "Apr 26, 1999" "Apr 26, 2000" "Apr 26, 2000"
 [301] "Apr 26, 2004" "Apr 26, 2006" "Apr 26, 2007" "Apr 26, 2011" "Apr 26, 2013" "Apr 26, 2013" "Apr 26, 2016" "Apr 26, 2016" "Apr 26, 2016" "Apr 26, 2016"
 [311] "Apr 26, 2016" "Apr 27, 1995" "Apr 27, 1997" "Apr 27, 1998" "Apr 27, 2000" "Apr 27, 2004" "Apr 27, 2004" "Apr 27, 2013" "Apr 27, 2015" "Apr 27, 2016"
 [321] "Apr 27, 2016" "Apr 27, 2016" "Apr 27, 2016" "Apr 27, 2016" "Apr 27, 2017" "Apr 27, 2019" "Apr 28, 1999" "Apr 28, 1999" "Apr 28, 1999" "Apr 28, 2000"
 [331] "Apr 28, 2000" "Apr 28, 2001" "Apr 28, 2001" "Apr 28, 2001" "Apr 28, 2004" "Apr 28, 2006" "Apr 28, 2010" "Apr 28, 2012" "Apr 28, 2015" "Apr 28, 2016"
 [341] "Apr 29, 1994" "Apr 29, 1997" "Apr 29, 1998" "Apr 29, 1998" "Apr 29, 2002" "Apr 29, 2003" "Apr 29, 2008" "Apr 29, 2013" "Apr 29, 2014" "Apr 29, 2014"
 [351] "Apr 29, 2014" "Apr 29, 2015" "Apr 29, 2015" "Apr 29, 2015" "Apr 29, 2016" "Apr 29, 2018" "Apr 29, 2018" "Apr 3, 1996"  "Apr 3, 2000"  "Apr 3, 2004" 
 [361] "Apr 3, 2008"  "Apr 3, 2014"  "Apr 3, 2014"  "Apr 3, 2015"  "Apr 3, 2016"  "Apr 3, 2019"  "Apr 3, 2019"  "Apr 3, 2019"  "Apr 30, 1994" "Apr 30, 1997"
 [371] "Apr 30, 1998" "Apr 30, 2000" "Apr 30, 2000" "Apr 30, 2002" "Apr 30, 2005" "Apr 30, 2010" "Apr 30, 2010" "Apr 30, 2014" "Apr 30, 2015" "Apr 30, 2015"
 [381] "Apr 30, 2017" "Apr 30, 2019" "Apr 4, 1994"  "Apr 4, 1996"  "Apr 4, 1998"  "Apr 4, 1999"  "Apr 4, 2000"  "Apr 4, 2001"  "Apr 4, 2007"  "Apr 4, 2014" 
 [391] "Apr 4, 2014"  "Apr 4, 2014"  "Apr 4, 2015"  "Apr 4, 2018"  "Apr 5, 1995"  "Apr 5, 1997"  "Apr 5, 1998"  "Apr 5, 1998"  "Apr 5, 1999"  "Apr 5, 2002" 
 [401] "Apr 5, 2006"  "Apr 5, 2007"  "Apr 5, 2009"  "Apr 5, 2010"  "Apr 5, 2016"  "Apr 5, 2017"  "Apr 5, 2018"  "Apr 5, 2019"  "Apr 6, 1998"  "Apr 6, 1998" 
 [411] "Apr 6, 2001"  "Apr 6, 2005"  "Apr 6, 2006"  "Apr 6, 2008"  "Apr 6, 2009"  "Apr 6, 2009"  "Apr 6, 2010"  "Apr 6, 2010"  "Apr 6, 2011"  "Apr 6, 2011" 
 [421] "Apr 6, 2011"  "Apr 6, 2011"  "Apr 6, 2011"  "Apr 6, 2014"  "Apr 6, 2014"  "Apr 6, 2014"  "Apr 6, 2014"  "Apr 6, 2014"  "Apr 6, 2014"  "Apr 6, 2014" 
 [431] "Apr 6, 2015"  "Apr 6, 2016"  "Apr 6, 2019"  "Apr 7, 1997"  "Apr 7, 1998"  "Apr 7, 1999"  "Apr 7, 2002"  "Apr 7, 2004"  "Apr 7, 2006"  "Apr 7, 2007" 
 [441] "Apr 7, 2008"  "Apr 7, 2008"  "Apr 7, 2009"  "Apr 7, 2009"  "Apr 7, 2014"  "Apr 7, 2015"  "Apr 7, 2019"  "Apr 8, 1994"  "Apr 8, 1994"  "Apr 8, 1997" 
 [451] "Apr 8, 1998"  "Apr 8, 1998"  "Apr 8, 1998"  "Apr 8, 1999"  "Apr 8, 1999"  "Apr 8, 1999"  "Apr 8, 2000"  "Apr 8, 2004"  "Apr 8, 2005"  "Apr 8, 2008" 
 [461] "Apr 8, 201 4" "Apr 8, 2010"  "Apr 8, 2010"  "Apr 8, 2014"  "Apr 8, 2014"  "Apr 8, 2015"  "Apr 8, 2015"  "Apr 8, 2016"  "Apr 8, 2018"  "Apr 8, 2019" 
 [471] "Apr 9, 1996"  "Apr 9, 1997"  "Apr 9, 1998"  "Apr 9, 1999"  "Apr 9, 1999"  "Apr 9, 1999"  "Apr 9, 2002"  "Apr 9, 2005"  "Apr 9, 2006"  "Apr 9, 2007" 
 [481] "Apr 9, 2011"  "Apr 9, 2013"  "Apr 9, 2015"  "Apr 9, 2016"  "Apr 9, 2016"  "Apr 9, 2016"  "Apr 9, 2018"  "Apr 9, 2019"  "Aug 03, 1996" "Aug 1, 1994" 
 [491] "Aug 1, 1994"  "Aug 1, 1998"  "Aug 1, 1998"  "Aug 1, 2001"  "Aug 1, 2003"  "Aug 1, 2005"  "Aug 1, 2006"  "Aug 1, 2007"  "Aug 1, 2008"  "Aug 1, 2008" 
 [501] "Aug 1, 2008"  "Aug 1, 2009"  "Aug 1, 2009"  "Aug 1, 2010"  "Aug 1, 2010"  "Aug 1, 2012"  "Aug 1, 2014"  "Aug 1, 2017"  "Aug 1, 2017"  "Aug 1, 2018" 
 [511] "Aug 1, 2018"  "Aug 1, 2019"  "Aug 1, 2019"  "Aug 1, 2019"  "Aug 10, 1995" "Aug 10, 1999" "Aug 10, 1999" "Aug 10, 1999" "Aug 10, 2000" "Aug 10, 2003"
 [521] "Aug 10, 2004" "Aug 10, 2004" "Aug 10, 2005" "Aug 10, 2006" "Aug 10, 2010" "Aug 10, 2014" "Aug 10, 2015" "Aug 10, 2016" "Aug 10, 2016" "Aug 10, 2017"
 [531] "Aug 11, 1994" "Aug 11, 1995" "Aug 11, 1996" "Aug 11, 1996" "Aug 11, 1997" "Aug 11, 1998" "Aug 11, 1999" "Aug 11, 2000" "Aug 11, 2005" "Aug 11, 2006"
 [541] "Aug 11, 2008" "Aug 11, 2010" "Aug 11, 2011" "Aug 11, 2011" "Aug 11, 2012" "Aug 11, 2016" "Aug 11, 2017" "Aug 11, 2017" "Aug 12, 1993" "Aug 12, 1996"
 [551] "Aug 12, 1997" "Aug 12, 1998" "Aug 12, 1998" "Aug 12, 1999" "Aug 12, 1999" "Aug 12, 2002" "Aug 12, 2004" "Aug 12, 2008" "Aug 12, 2014" "Aug 12, 2015"
 [561] "Aug 12, 2017" "Aug 13, 1993" "Aug 13, 1999" "Aug 13, 1999" "Aug 13, 2000" "Aug 13, 2004" "Aug 13, 2004" "Aug 13, 2005" "Aug 13, 2008" "Aug 13, 2010"
 [571] "Aug 13, 2012" "Aug 13, 2013" "Aug 13, 2015" "Aug 13, 2018" "Aug 13, 2018" "Aug 13, 2018" "Aug 14, 1996" "Aug 14, 2001" "Aug 14, 2001" "Aug 14, 2002"
 [581] "Aug 14, 2002" "Aug 14, 2003" "Aug 14, 2008" "Aug 14, 2010" "Aug 14, 2014" "Aug 14, 2014" "Aug 14, 2014" "Aug 14, 2014" "Aug 14, 2014" "Aug 14, 2014"
 [591] "Aug 14, 2014" "Aug 14, 2014" "Aug 14, 2016" "Aug 14, 2017" "Aug 14, 2017" "Aug 14, 2018" "Aug 15, 1996" "Aug 15, 1997" "Aug 15, 2002" "Aug 15, 2004"
 [601] "Aug 15, 2004" "Aug 15, 2007" "Aug 15, 2008" "Aug 15, 2008" "Aug 15, 2009" "Aug 15, 2011" "Aug 15, 2012" "Aug 15, 2013" "Aug 15, 2013" "Aug 15, 2013"
 [611] "Aug 15, 2014" "Aug 15, 2014" "Aug 15, 2016" "Aug 15, 2018" "Aug 15, 2019" "Aug 16, 1997" "Aug 16, 1999" "Aug 16, 2000" "Aug 16, 2001" "Aug 16, 2003"
 [621] "Aug 16, 2003" "Aug 16, 2005" "Aug 16, 2006" "Aug 16, 2010" "Aug 16, 2010" "Aug 16, 2012" "Aug 16, 2015" "Aug 16, 2015" "Aug 16, 2017" "Aug 16, 2017"
 [631] "Aug 16, 2017" "Aug 16, 2017" "Aug 16, 2019" "Aug 16, 2019" "Aug 16, 2019" "Aug 17, 2000" "Aug 17, 2001" "Aug 17, 2009" "Aug 17, 2011" "Aug 17, 2011"
 [641] "Aug 17, 2011" "Aug 17, 2012" "Aug 17, 2014" "Aug 17, 2014" "Aug 17, 2016" "Aug 17, 2017" "Aug 17, 2019" "Aug 18, 1998" "Aug 18, 1999" "Aug 18, 1999"
 [651] "Aug 18, 1999" "Aug 18, 2000" "Aug 18, 2008" "Aug 18, 2011" "Aug 18, 2014" "Aug 18, 2016" "Aug 18, 2017" "Aug 18, 2019" "Aug 18, 2019" "Aug 18, 2019"
 [661] "Aug 18, 2019" "Aug 19, 1998" "Aug 19, 1998" "Aug 19, 1998" "Aug 19, 1998" "Aug 19, 1998" "Aug 19, 2000" "Aug 19, 2000" "Aug 19, 2002" "Aug 19, 2004"
 [671] "Aug 19, 2005" "Aug 19, 2008" "Aug 19, 2009" "Aug 19, 2009" "Aug 19, 2010" "Aug 19, 2010" "Aug 19, 2010" "Aug 19, 2011" "Aug 19, 2011" "Aug 19, 2012"
 [681] "Aug 19, 2014" "Aug 19, 2014" "Aug 19, 2015" "Aug 19, 2016" "Aug 19, 2016" "Aug 19, 2019" "Aug 19, 2019" "Aug 19, 2019" "Aug 19, 2019" "Aug 2, 1997" 
 [691] "Aug 2, 1997"  "Aug 2, 1998"  "Aug 2, 2000"  "Aug 2, 2000"  "Aug 2, 2000"  "Aug 2, 2003"  "Aug 2, 2003"  "Aug 2, 2004"  "Aug 2, 2007"  "Aug 2, 2008" 
 [701] "Aug 2, 2014"  "Aug 2, 2015"  "Aug 2, 2015"  "Aug 2, 2015"  "Aug 2, 2017"  "Aug 2, 2018"  "Aug 20, 1996" "Aug 20, 1996" "Aug 20, 1996" "Aug 20, 1997"
 [711] "Aug 20, 1997" "Aug 20, 1998" "Aug 20, 1998" "Aug 20, 1999" "Aug 20, 2003" "Aug 20, 2003" "Aug 20, 2005" "Aug 20, 2006" "Aug 20, 2012" "Aug 20, 2013"
 [721] "Aug 20, 2014" "Aug 20, 2014" "Aug 20, 2015" "Aug 20, 2019" "Aug 20, 2019" "Aug 21, 1993" "Aug 21, 1996" "Aug 21, 1997" "Aug 21, 1997" "Aug 21, 1999"
 [731] "Aug 21, 1999" "Aug 21, 2000" "Aug 21, 2000" "Aug 21, 2003" "Aug 21, 2006" "Aug 21, 2008" "Aug 21, 2008" "Aug 21, 2009" "Aug 21, 2013" "Aug 21, 2014"
 [741] "Aug 21, 2018" "Aug 21, 2018" "Aug 21, 2018" "Aug 21, 2018" "Aug 21, 2019" "Aug 21, 2019" "Aug 21, 2019" "Aug 21, 2019" "Aug 21, 2019" "Aug 22, 1994"
 [751] "Aug 22, 1998" "Aug 22, 1998" "Aug 22, 2001" "Aug 22, 2003" "Aug 22, 2008" "Aug 22, 2011" "Aug 22, 2016" "Aug 22, 2018" "Aug 22, 2018" "Aug 22, 2018"
 [761] "Aug 22, 2019" "Aug 23, 1996" "Aug 23, 1997" "Aug 23, 1997" "Aug 23, 1998" "Aug 23, 1999" "Aug 23, 2000" "Aug 23, 2001" "Aug 23, 2004" "Aug 23, 2005"
 [771] "Aug 23, 2005" "Aug 23, 2005" "Aug 23, 2006" "Aug 23, 2006" "Aug 23, 2006" "Aug 23, 2007" "Aug 23, 2010" "Aug 23, 2013" "Aug 23, 2013" "Aug 23, 2016"
 [781] "Aug 23, 2016" "Aug 23, 2018" "Aug 23, 2019" "Aug 23, 2019" "Aug 23, 2019" "Aug 24, 1993" "Aug 24, 1994" "Aug 24, 1997" "Aug 24, 1998" "Aug 24, 1998"
 [791] "Aug 24, 1999" "Aug 24, 1999" "Aug 24, 2000" "Aug 24, 2000" "Aug 24, 2002" "Aug 24, 2005" "Aug 24, 2005" "Aug 24, 2005" "Aug 24, 2007" "Aug 24, 2008"
 [801] "Aug 24, 2008" "Aug 24, 2009" "Aug 24, 2011" "Aug 24, 2013" "Aug 24, 2014" "Aug 24, 2016" "Aug 24, 2017" "Aug 24, 2017" "Aug 24, 2017" "Aug 24, 2017"
 [811] "Aug 24, 2019" "Aug 24, 2019" "Aug 25, 1995" "Aug 25, 1997" "Aug 25, 1998" "Aug 25, 1998" "Aug 25, 1999" "Aug 25, 2003" "Aug 25, 2003" "Aug 25, 2005"
 [821] "Aug 25, 2006" "Aug 25, 2008" "Aug 25, 2008" "Aug 25, 2009" "Aug 25, 2012" "Aug 25, 2016" "Aug 25, 2016" "Aug 25, 2016" "Aug 26, 1995" "Aug 26, 1997"
 [831] "Aug 26, 1998" "Aug 26, 2000" "Aug 26, 2001" "Aug 26, 2003" "Aug 26, 2008" "Aug 26, 2008" "Aug 26, 2008" "Aug 26, 2009" "Aug 26, 2009" "Aug 26, 2013"
 [841] "Aug 26, 2014" "Aug 26, 2016" "Aug 26, 2018" "Aug 27, 1997" "Aug 27, 1997" "Aug 27, 1999" "Aug 27, 2002" "Aug 27, 2003" "Aug 27, 2005" "Aug 27, 2008"
 [851] "Aug 27, 2014" "Aug 27, 2014" "Aug 27, 2015" "Aug 27, 2017" "Aug 27, 2019" "Aug 28, 1995" "Aug 28, 1996" "Aug 28, 1997" "Aug 28, 1999" "Aug 28, 1999"
 [861] "Aug 28, 2000" "Aug 28, 2002" "Aug 28, 2005" "Aug 28, 2005" "Aug 28, 2007" "Aug 28, 2008" "Aug 28, 2011" "Aug 28, 2011" "Aug 28, 2011" "Aug 28, 2012"
 [871] "Aug 28, 2012" "Aug 28, 2013" "Aug 28, 2014" "Aug 28, 2014" "Aug 28, 2014" "Aug 28, 2016" "Aug 28, 2016" "Aug 28, 2017" "Aug 28, 2018" "Aug 28, 2018"
 [881] "Aug 29, 1994" "Aug 29, 2000" "Aug 29, 2002" "Aug 29, 2005" "Aug 29, 2005" "Aug 29, 2007" "Aug 29, 2010" "Aug 29, 2012" "Aug 29, 2012" "Aug 29, 2013"
 [891] "Aug 29, 2014" "Aug 29, 2016" "Aug 29, 2017" "Aug 29, 2017" "Aug 29, 2018" "Aug 3, 1999"  "Aug 3, 2004"  "Aug 3, 2004"  "Aug 3, 2005"  "Aug 3, 2005" 
 [901] "Aug 3, 2005"  "Aug 3, 2007"  "Aug 3, 2008"  "Aug 3, 2008"  "Aug 3, 2008"  "Aug 3, 2008"  "Aug 3, 2010"  "Aug 3, 2014"  "Aug 3, 2014"  "Aug 3, 2014" 
 [911] "Aug 3, 2017"  "Aug 30, 1994" "Aug 30, 1998" "Aug 30, 2000" "Aug 30, 2003" "Aug 30, 2004" "Aug 30, 2005" "Aug 30, 2006" "Aug 30, 2007" "Aug 30, 2007"
 [921] "Aug 30, 2008" "Aug 30, 2009" "Aug 30, 2010" "Aug 30, 2015" "Aug 30, 2015" "Aug 30, 2015" "Aug 30, 2015" "Aug 30, 2015" "Aug 30, 2016" "Aug 30, 2017"
 [931] "Aug 30, 2017" "Aug 30, 2018" "Aug 30, 2018" "Aug 31, 1995" "Aug 31, 1999" "Aug 31, 2000" "Aug 31, 2004" "Aug 31, 2005" "Aug 31, 2005" "Aug 31, 2009"
 [941] "Aug 31, 2011" "Aug 31, 2013" "Aug 31, 2014" "Aug 31, 2015" "Aug 31, 2016" "Aug 31, 2016" "Aug 31, 2017" "Aug 31, 2017" "Aug 31, 2018" "Aug 4, 1995" 
 [951] "Aug 4, 1995"  "Aug 4, 1999"  "Aug 4, 2003"  "Aug 4, 2003"  "Aug 4, 2004"  "Aug 4, 2005"  "Aug 4, 2005"  "Aug 4, 2006"  "Aug 4, 2006"  "Aug 4, 2007" 
 [961] "Aug 4, 2008"  "Aug 4, 2012"  "Aug 4, 2017"  "Aug 4, 2017"  "Aug 5, 1994"  "Aug 5, 1995"  "Aug 5, 1998"  "Aug 5, 1999"  "Aug 5, 2002"  "Aug 5, 2003" 
 [971] "Aug 5, 2005"  "Aug 5, 2005"  "Aug 5, 2010"  "Aug 5, 2010"  "Aug 5, 2015"  "Aug 5, 2015"  "Aug 5, 2017"  "Aug 5, 2017"  "Aug 5, 2019"  "Aug 5, 2019" 
 [981] "Aug 5, 2019"  "Aug 6, 1993"  "Aug 6, 1997"  "Aug 6, 1998"  "Aug 6, 2001"  "Aug 6, 2002"  "Aug 6, 2003"  "Aug 6, 2004"  "Aug 6, 2006"  "Aug 6, 2006" 
 [991] "Aug 6, 2014"  "Aug 6, 2015"  "Aug 6, 2015"  "Aug 6, 2016"  "Aug 6, 2016"  "Aug 6, 2017"  "Aug 6, 2017"  "Aug 6, 2017"  "Aug 6, 2017"  "Aug 7, 1993" 
 [ reached getOption("max.print") -- omitted 5461 entries ]
numPreDeDup<-nrow(pq_metaclean) #6461
print(paste("Number of articles pre-dedup:",numPreDeDup))
[1] "Number of articles pre-dedup: 6461"
numDup<-sum(table(pq_metaclean$`Full text`)-1) # Total number of duplicate full texts: 184
print(paste("Number of duplicates:",numDup))
[1] "Number of duplicates: 184"
# duplicate flag is not sufficient to remove all duplicates - 53 rows
pq_metadata %>%
  select(`Full text`, `Publication info`, `Publication date`) %>%
  mutate(`Publication date` = as.Date(`Publication date`, format = "%b %d, %Y")) %>%
  arrange(`Publication date`) %>%
  filter(grepl('Duplicate',`Publication info`))

pq_metaclean <- pq_metaclean %>%
  filter(!grepl('Duplicate',`Publication info`)) %>%
  group_by(`Full text`) %>% 
  mutate(`Publication date` = as.Date(`Publication date`, format = "%b %d, %Y")) %>%
  arrange(`Publication date`) %>%
  slice(1L)

pq_metaclean %>%
  select(`Publication info`,`Publication date`)
Adding missing grouping variables: `Full text`
numPostDeDup<-nrow(pq_metaclean)
print(paste("Number of articles post-dedup:",numPostDeDup)) # 6277 to 6239
[1] "Number of articles post-dedup: 6239"
numRemoved <- (numPreDeDup - numPostDeDup)
print(paste("Number of duplicates removed:",numRemoved)) # 184 to 222
[1] "Number of duplicates removed: 222"

Save to pq_metaclean.csv if it doesn’t already exist or overwrite = TRUE

# Save cleaned corpus to new file 
outputFileNameClean <- paste0(rFileNum,"_pq_metaclean")
outputFolder = "Data/02_Working/"
outputFileClean = paste(outputFolder,outputFileNameClean,".csv",sep="")

if (!file.exists(outputFileClean) | overwrite) {
  write.csv(pq_metaclean, outputFileClean, row.names=FALSE)
}
LS0tDQp0aXRsZTogInBxX2NsZWFuIg0Kb3V0cHV0OiBodG1sX25vdGVib29rDQplZGl0b3Jfb3B0aW9uczogDQogIGNodW5rX291dHB1dF90eXBlOiBpbmxpbmUNCi0tLQ0KIyBTVEVQIDEgLSBDbGVhbiBhbmQgaWRlbnRpZnkgbWlzdGFrZXMgZnJvbSBwcV9wYXJzZXIgc2NyaXB0DQoNClRoaXMgbm90ZWJvb2s6DQoxLiBSZWFkcyBpbiB0aGUgb3V0cHV0IGZyb20gcHFfcGFyc2VyLmlweW5iICgicHFfbWV0YWRhdGEuY3N2IikNCjIuIENsZWFucyB0aGUgb3V0cHV0IGZyb20gcHFfcGFyc2VyLmlweW5iDQozLiBXcml0ZXMgY2xlYW5lZCBjb3JwdXMgKCIwMV9wcV9tZXRhY2xlYW4uY3N2IikNCg0KYGBge3IsIGVjaG89RkFMU0UsIHJlc3VsdHM9ImhpZGUiLCBtZXNzYWdlcz1GQUxTRX0NCiMgTG9hZCBhbmQgSW5zdGFsbCBMaWJyYXJpZXMNCnNvdXJjZSgiU0VOX2Z1bmN0aW9ucy5SIikNCiMjIENoZWNrIGxpYnJhcmllcyAmIGluc3RhbGwNCkxpYnJhcnlMaXN0PC1jKCJzdHJpbmdyIiwiZGF0YS50YWJsZSIsImRwbHlyIiwidGlkeXIiLCJtYWdyaXR0ciIsIk5MUCIsInRpZHl0ZXh0IiwidG0iLCJnZ3Bsb3QyIiwNCiAgICAgICAgICAgICAgICJzY2FsZXMiLCAiZ2d3b3JkY2xvdWQiLCJ0ZXh0bWluZVIiLCJkaWdlc3QiLCAicnZlc3QiLCJ0ZXh0Y2xlYW4iLCJmdXR1cmUuYXBwbHkiKQ0KaW5zdGFsbF9vcl9sb2FkX3BhY2soTGlicmFyeUxpc3QpDQpyRmlsZU51bSA9ICIwMSINCm92ZXJ3cml0ZSA9IFRSVUUNCmBgYA0KDQojIGxvYWQgaW4gZGF0YQ0KYGBge3J9DQojbG9hZCBkYXRhDQpwcV9tZXRhZGF0YSA8LSBkYXRhLnRhYmxlOjpmcmVhZCgnRGF0YS8wMl9Xb3JraW5nL3BxX21ldGFkYXRhLmNzdicpDQoNCiMgZGlzcGxheXMgY29sdW1uIG5hbWVzICYgbnVtYmVyIG9mIHJvd3MgLSA2NTg5DQpuYW1lcyhwcV9tZXRhZGF0YSk7bnJvdyhwcV9tZXRhZGF0YSkNCg0KIyBIZWFkIGRpc3BsYXlzIHRoZSBmaXJzdCA2IHJvd3Mgb2YgdGhlIGRhdGEudGFibGUNCmhlYWQocHFfbWV0YWRhdGEpDQpgYGANCg0KIyBjaGVjayBlbXB0eSBmdWxsIHRleHRzDQpgYGB7cn0NCiMgSG93IGRvIHlvdSB3YW50IHRvIHRyZWF0IE5BPw0KIyBzdWJzZXQgdGhlIGRhdGEgYnkgdGhlIE5BcywgZXhwbG9yZSB0aGVtICYgdGhlaXIgb3JpZ2luDQojIGNhbiB3ZSBwcm9ncmFtbWF0aWNhbGx5IGZpeCBpdCwgb3IgZG8gd2UgaGF2ZSB0byBkbyBpdCBtYW51YWxseT8NCg0KIyA1MSByb3dzIGJsYW5rL05BIEZ1bGwgdGV4dA0KcHFfZW1wdHk8LXBxX21ldGFkYXRhW3BxX21ldGFkYXRhJGBGdWxsIHRleHRgID09ICIiIHwgaXMubmEocHFfbWV0YWRhdGEkYEZ1bGwgdGV4dGApLF0NCm5yb3cocHFfZW1wdHkpDQpgYGANCg0KIyByZW1vdmUgZW1wdHkgZnVsbCB0ZXh0cyAmIG1pc3NpbmcgSURzDQpgYGB7cn0NCiMgSG93IGRvIHlvdSB3YW50IHRvIHRyZWF0IE5BPw0KIyBzdWJzZXQgdGhlIGRhdGEgYnkgdGhlIE5BcywgZXhwbG9yZSB0aGVtICYgdGhlaXIgb3JpZ2luDQojIGNhbiB3ZSBwcm9ncmFtbWF0aWNhbGx5IGZpeCBpdCwgb3IgZG8gd2UgaGF2ZSB0byBkbyBpdCBtYW51YWxseT8NCg0KIyA1MCBibGFuay9OQSBwdWIgdGl0bGVzDQojcHFfbWV0YWNsZWFuPC1wcV9tZXRhZGF0YVshKHBxX21ldGFkYXRhJGBGdWxsIHRleHRgID09ICIiIHwgaXMubmEocHFfbWV0YWRhdGEkYEZ1bGwgdGV4dGApKSxdDQoNCnBxX21ldGFjbGVhbiA8LSBwcV9tZXRhZGF0YSAlPiUgDQogIGRyb3BfbmEoYEZ1bGwgdGV4dGApICU+JQ0KICBkcm9wX25hKGBQcm9RdWVzdCBkb2N1bWVudCBJRGApICU+JQ0KICBmaWx0ZXIoYEZ1bGwgdGV4dGAgIT0gIiIpICU+JQ0KICBmaWx0ZXIoYFByb1F1ZXN0IGRvY3VtZW50IElEYCAhPSAiIikNCg0KbnVtUmVkdWNlZDwtIG5yb3cocHFfbWV0YWRhdGEpLW5yb3cocHFfbWV0YWNsZWFuKQ0KcHJpbnQocGFzdGUoIk5BIG9yIGJsYW5rIEZ1bGwgVGV4dCByb3dzIHJlbW92ZWQ6IixudW1SZWR1Y2VkKSkNCmBgYA0KDQojIGNoZWNrIHVuaXF1ZSBwdWJsaWNhdGlvbiB0aXRsZXMNCmBgYHtyfQ0KIyB3aGF0IGFyZSB0aGUgdW5pcXVlIHB1YmxpY2F0aW9uIHRpdGxlcyANCnVuaXF1ZShwcV9tZXRhY2xlYW4kYFB1YmxpY2F0aW9uIHRpdGxlYCkNCmBgYA0KDQojIGNoZWNrIGVtcHR5IHB1YmxpY2F0aW9uIHRpdGxlcw0KYGBge3J9DQojIEhvdyBkbyB5b3Ugd2FudCB0byB0cmVhdCBOQT8NCiMgc3Vic2V0IHRoZSBkYXRhIGJ5IHRoZSBOQXMsIGV4cGxvcmUgdGhlbSAmIHRoZWlyIG9yaWdpbg0KIyBjYW4gd2UgcHJvZ3JhbW1hdGljYWxseSBmaXggaXQsIG9yIGRvIHdlIGhhdmUgdG8gZG8gaXQgbWFudWFsbHk/DQoNCiMgMTUgYmxhbmsvTkEgcHViIHRpdGxlcyAodGhlcmUgd2VyZSA1MCBiZWZvcmUgRnVsbCB0ZXh0IGNsZWFuKQ0KcHFfZW1wdHk8LXBxX21ldGFjbGVhbltwcV9tZXRhY2xlYW4kYFB1YmxpY2F0aW9uIHRpdGxlYCA9PSAiIiB8IGlzLm5hKHBxX21ldGFjbGVhbiRgUHVibGljYXRpb24gdGl0bGVgKSxdDQpucm93KHBxX2VtcHR5KQ0KIyAwIG9mIHRoZXNlIGJsYW5rL05BIHB1YiB0aXRsZXMgaGF2ZSBibGFuay9OQSAnZnVsbCB0ZXh0cycgYXMgd2VsbCAodGhlcmUgd2VyZSAzNyBiZWZvcmUgZnVsbCB0ZXh0IGNsZWFuKQ0KbnJvdyhwcV9lbXB0eVtwcV9lbXB0eSRgRnVsbCB0ZXh0YCA9PSAiIiB8IGlzLm5hKHBxX2VtcHR5JGBGdWxsIHRleHRgKSxdKQ0KYGBgDQoNCiMgaWYgcHVibGljYXRpb24gdGl0bGUgaXMgYmxhbmsgb3IgTkEsIHJlcGxhY2UgaXQgd2l0aCB0aGUgcHVibGljYXRpb24gdGl0bGUgaW4gcHVibGljYXRpb24gaW5mbw0KYGBge3J9DQpwcV9tZXRhY2xlYW4gPC0gcHFfbWV0YWNsZWFuICU+JSANCiAgbXV0YXRlKGBQdWJsaWNhdGlvbiB0aXRsZWAgPSBpZmVsc2UoYFB1YmxpY2F0aW9uIHRpdGxlYCA9PSAiIiB8IGlzLm5hKGBQdWJsaWNhdGlvbiB0aXRsZWApLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdW5saXN0KHN0cnNwbGl0KGBQdWJsaWNhdGlvbiBpbmZvYCwgc3BsaXQ9IlxcW3w6IikpWzFdLCBgUHVibGljYXRpb24gdGl0bGVgKSkNCg0KI2NoZWNrIHRvIHNlZSBpZiBwdWIgdGl0bGUgc3Vic3RpdHV0aW9uIHdvcmtlZCANCnVuaXF1ZShwcV9tZXRhY2xlYW4kYFB1YmxpY2F0aW9uIHRpdGxlYCkNCmBgYA0KDQojIGZpeCBkdXBsaWNhdGUgcHVibGljYXRpb24gdGl0bGVzDQpgYGB7cn0NCiMgU3Vic3RpdHV0ZSANCiMgIkJhbmdvciBEYWlseSBOZXdzOyBCYW5nIG9yLCBNZS4iICANCiMgIkJhbmdvciBEYWlseSBOZXdzOyBCYSBuZ29yLCBNZS4iIA0KIyAiQmFuZ29yIERhaWwgeSBOZXdzOyBCYW5nb3IsIE1lLiIgIA0KIyB3aXRoICJCYW5nb3IgRGFpbHkgTmV3czsgQmFuZ29yLCBNZS4iICANCg0KIyAiTW9ybmluZyBTZW50aW5lbDsgV2F0ZSBydmlsbGUsIE1lLiIgDQojICJDZW50cmFsIE1haW5lIE1vcm5pbmcgU2VudGluZWw7IFdhdGVydmlsbGUsIE1lLiINCiMgIHdpdGggIk1vcm5pbmcgU2VudGluZWw7IFdhdGVydmlsbGUsIE1lLiIgICAgICAgDQoNCiMgIktlbm5lYmVjIEpvdXJuYWw7IEF1Z3VzdGEsIE1lLiIgDQojICJLZW5uZWJlYyBKb3VybmFsOyBBdWd1c3RhLCBNZSAuIg0KDQpwcV9tZXRhY2xlYW4gPC0gcHFfbWV0YWNsZWFuICU+JSANCiAgbXV0YXRlKGBQdWJsaWNhdGlvbiB0aXRsZWAgPSByZWNvZGUoYFB1YmxpY2F0aW9uIHRpdGxlYCwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ0JhbmdvciBEYWlseSBOZXdzOyBCYW5nIG9yLCBNZS4nID0gJ0JhbmdvciBEYWlseSBOZXdzOyBCYW5nb3IsIE1lLicsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdCYW5nb3IgRGFpbHkgTmV3czsgQmEgbmdvciwgTWUuJyA9ICdCYW5nb3IgRGFpbHkgTmV3czsgQmFuZ29yLCBNZS4nLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnQmFuZ29yIERhaWwgeSBOZXdzOyBCYW5nb3IsIE1lLicgPSAnQmFuZ29yIERhaWx5IE5ld3M7IEJhbmdvciwgTWUuJywNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ0JhbmdvciBEYWlseSBOZXdzOyBCYW5nbyByLCBNZS4nID0gJ0JhbmdvciBEYWlseSBOZXdzOyBCYW5nb3IsIE1lLicsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdCYW5nIG9yIERhaWx5IE5ld3M7IEJhbmdvciwgTWUuJyA9ICdCYW5nb3IgRGFpbHkgTmV3czsgQmFuZ29yLCBNZS4nLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiQmFuZ29yIERhaWx5IE5ld3MgOyBCYW5nb3IsIE1lLiAiID0gJ0JhbmdvciBEYWlseSBOZXdzOyBCYW5nb3IsIE1lLicsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdNb3JuaW5nIFNlbnRpbmVsOyBXYXRlIHJ2aWxsZSwgTWUuJyA9ICdNb3JuaW5nIFNlbnRpbmVsOyBXYXRlcnZpbGxlLCBNZS4nLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnQ2VudHJhbCBNYWluZSBNb3JuaW5nIFNlbnRpbmVsOyBXYXRlcnZpbGxlLCBNZS4nID0gJ01vcm5pbmcgU2VudGluZWw7IFdhdGVydmlsbGUsIE1lLicsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdLZW5uZWJlYyBKb3VybmFsOyBBdWd1c3RhLCBNZSAuJyA9ICdLZW5uZWJlYyBKb3VybmFsOyBBdWd1c3RhLCBNZS4nLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnUG9ydGxhbmQgUHJlc3MgSGVyYWxkOyBQb3J0IGxhbmQsIE1lLicgPSAnUG9ydGxhbmQgUHJlc3MgSGVyYWxkOyBQb3J0bGFuZCwgTWUuJywNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ1BvcnRsYW5kIFByIGVzcyBIZXJhbGQ7IFBvcnRsYW5kLCBNZS4nID0gJ1BvcnRsYW5kIFByZXNzIEhlcmFsZDsgUG9ydGxhbmQsIE1lLicsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdQb3J0bGFuZCBQcmUgc3MgSGVyYWxkOyBQb3J0bGFuZCwgTWUuJyA9ICdQb3J0bGFuZCBQcmVzcyBIZXJhbGQ7IFBvcnRsYW5kLCBNZS4nLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnUG9ydGwgYW5kIFByZXNzIEhlcmFsZDsgUG9ydGxhbmQsIE1lLicgPSAnUG9ydGxhbmQgUHJlc3MgSGVyYWxkOyBQb3J0bGFuZCwgTWUuJywNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ1BvcnRsYW5kIFByZXNzIEhlcmFsZCBQb3J0bGFuZCwgTWUuJyA9ICdQb3J0bGFuZCBQcmVzcyBIZXJhbGQ7IFBvcnRsYW5kLCBNZS4nLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnUG9ydGxhbiBkIFByZXNzIEhlcmFsZDsgUG9ydGxhbmQsIE1lLicgPSAnUG9ydGxhbmQgUHJlc3MgSGVyYWxkOyBQb3J0bGFuZCwgTWUuJywNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ1BvcnQgbGFuZCBQcmVzcyBIZXJhbGQ7IFBvcnRsYW5kLCBNZS4nID0gJ1BvcnRsYW5kIFByZXNzIEhlcmFsZDsgUG9ydGxhbmQsIE1lLicsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdQb3J0bGFuIGQgUHJlc3MgSGVyYWxkOyBQb3J0bGFuZCwgTWUuJyA9ICdQb3J0bGFuZCBQcmVzcyBIZXJhbGQ7IFBvcnRsYW5kLCBNZS4nLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnUG9ydCBsYW5kIFByZXNzIEhlcmFsZDsgUG9ydGxhbmQsIE1lLicgPSAnUG9ydGxhbmQgUHJlc3MgSGVyYWxkOyBQb3J0bGFuZCwgTWUuJykpDQoNCiNjaGVjayB0byBzZWUgaWYgcHViIHRpdGxlIHN1YnN0aXR1dGlvbiB3b3JrZWQgLS0tIHllcywgbm8gd2VpcmQgcmVwZWF0cw0KdW5pcXVlKHBxX21ldGFjbGVhbiRgUHVibGljYXRpb24gdGl0bGVgKQ0KYGBgDQoNCiNjaGVjayB1bmlxdWUgUHVibGljYXRpb24geWVhcnMNCmBgYHtyfQ0KdW5pcXVlKHBxX21ldGFjbGVhbiQnUHVibGljYXRpb24geWVhcicpDQpgYGANCg0KI2NsZWFuIHB1YmxpY2F0aW9uIHllYXJzDQpgYGB7cn0NCnBxX21ldGFjbGVhbiA8LSBwcV9tZXRhY2xlYW4gJT4lDQogIG11dGF0ZShgUHVibGljYXRpb24geWVhcmAgPSByZWNvZGUoYFB1YmxpY2F0aW9uIHllYXJgLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICcyMDAgMycgPSAnMjAwMycsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJzIwMSA4JyA9ICcyMDE4JykpDQoNCiNjaGVjayB0byBzZWUgaWYgcHViIHRpdGxlIHN1YnN0aXR1dGlvbiB3b3JrZWQgLS0tIHllcywgbm8gd2VpcmQgcmVwZWF0cw0KdW5pcXVlKHBxX21ldGFjbGVhbiRgUHVibGljYXRpb24geWVhcmApDQpgYGANCg0KIyBjaGVjayBlbXB0eSBwdWJsaWNhdGlvbiB5ZWFycw0KYGBge3J9DQojIEhvdyBkbyB5b3Ugd2FudCB0byB0cmVhdCBOQT8NCiMgc3Vic2V0IHRoZSBkYXRhIGJ5IHRoZSBOQXMsIGV4cGxvcmUgdGhlbSAmIHRoZWlyIG9yaWdpbg0KIyBjYW4gd2UgcHJvZ3JhbW1hdGljYWxseSBmaXggaXQsIG9yIGRvIHdlIGhhdmUgdG8gZG8gaXQgbWFudWFsbHk/DQoNCiMgMyBibGFuay9OQSBwdWIgeWVhcnMNCnBxX2VtcHR5PC1wcV9tZXRhY2xlYW5bcHFfbWV0YWNsZWFuJGBQdWJsaWNhdGlvbiB5ZWFyYCA9PSAiIiB8IGlzLm5hKHBxX21ldGFjbGVhbiRgUHVibGljYXRpb24geWVhcmApLF0NCm5yb3cocHFfZW1wdHkpDQojIDAgb2YgdGhlc2UgYmxhbmsvTkEgcHViIHllYXJzIGhhdmUgYmxhbmsvTkEgJ2Z1bGwgdGV4dHMnIGFzIHdlbGwNCm5yb3cocHFfZW1wdHlbcHFfZW1wdHkkYEZ1bGwgdGV4dGAgPT0gIiIgfCBpcy5uYShwcV9lbXB0eSRgRnVsbCB0ZXh0YCksXSkNCmBgYA0KDQojIGlmIHB1YmxpY2F0aW9uIHllYXIgaXMgYmxhbmsgb3IgTkEsIHJlcGxhY2UgaXQgd2l0aCB0aGUgeWVhciBpbiBwdWJsaWNhdGlvbiBpbmZvDQpgYGB7cn0NCiNwcV9tZXRhY2xlYW4gPC0gcHFfbWV0YWNsZWFuICU+JQ0KIyAgbXV0YXRlKGBQdWJsaWNhdGlvbiB5ZWFyYCA9IGlmZWxzZShgUHVibGljYXRpb24geWVhcmAgPT0gIiIgfCBpcy5uYShgUHVibGljYXRpb24geWVhcmApLCBzdHJfc3ViKGBQdWJsaWNhdGlvbiBkYXRlYCwtNCwtMSksIGBQdWJsaWNhdGlvbiB5ZWFyYCkpDQoNCiMgU3ViIGZyb20gbGFzdCA0IGRpZ2l0cyBpbiBwdWJsaWNhdGlvbiBkYXRlDQojIHN1YiBmcm9tIG1hdGNoZWQgY29uZGl0aW9uIG9mIGJlaW5nIDQgZGlnaXRzIGluIHB1YmxpY2F0aW9uIGluZm8NCnBxX21ldGFjbGVhbiA8LSBwcV9tZXRhY2xlYW4gJT4lDQogIG11dGF0ZShgUHVibGljYXRpb24geWVhcmAgPSBpZmVsc2UoYFB1YmxpY2F0aW9uIHllYXJgID09ICIiIHwgaXMubmEoYFB1YmxpY2F0aW9uIHllYXJgKSwgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3ViKCcuKihcXGR7NH0pLionLCAnXFwxJywgYFB1YmxpY2F0aW9uIGluZm9gKSwgYFB1YmxpY2F0aW9uIHllYXJgKSkNCiNjaGVjayB0byBzZWUgaWYgcHViIHRpdGxlIHN1YnN0aXR1dGlvbiB3b3JrZWQgLS0tIHllcywgbm8gd2VpcmQgcmVwZWF0cw0KdW5pcXVlKHBxX21ldGFjbGVhbiRgUHVibGljYXRpb24geWVhcmApDQpgYGANCg0KIyByZS1jaGVjayBlbXB0eSBwdWJsaWNhdGlvbiB5ZWFycyAoc2hvdWxkIGJlIDAgbm93KQ0KYGBge3J9DQojIEhvdyBkbyB5b3Ugd2FudCB0byB0cmVhdCBOQT8NCiMgc3Vic2V0IHRoZSBkYXRhIGJ5IHRoZSBOQXMsIGV4cGxvcmUgdGhlbSAmIHRoZWlyIG9yaWdpbg0KIyBjYW4gd2UgcHJvZ3JhbW1hdGljYWxseSBmaXggaXQsIG9yIGRvIHdlIGhhdmUgdG8gZG8gaXQgbWFudWFsbHk/DQoNCiMgMyBibGFuay9OQSBwdWIgeWVhcnMNCnBxX2VtcHR5PC1wcV9tZXRhY2xlYW5bcHFfbWV0YWNsZWFuJGBQdWJsaWNhdGlvbiB5ZWFyYCA9PSAiIiB8IGlzLm5hKHBxX21ldGFjbGVhbiRgUHVibGljYXRpb24geWVhcmApLF0NCm5yb3cocHFfZW1wdHkpDQojIDAgb2YgdGhlc2UgYmxhbmsvTkEgcHViIHllYXJzIGhhdmUgYmxhbmsvTkEgJ2Z1bGwgdGV4dHMnIGFzIHdlbGwNCm5yb3cocHFfZW1wdHlbcHFfZW1wdHkkYEZ1bGwgdGV4dGAgPT0gIiIgfCBpcy5uYShwcV9lbXB0eSRgRnVsbCB0ZXh0YCksXSkNCmBgYA0KDQojIGNoZWNrIGVtcHR5IHB1YmxpY2F0aW9uIGRhdGVzDQpgYGB7cn0NCiMgSG93IGRvIHlvdSB3YW50IHRvIHRyZWF0IE5BPw0KIyBzdWJzZXQgdGhlIGRhdGEgYnkgdGhlIE5BcywgZXhwbG9yZSB0aGVtICYgdGhlaXIgb3JpZ2luDQojIGNhbiB3ZSBwcm9ncmFtbWF0aWNhbGx5IGZpeCBpdCwgb3IgZG8gd2UgaGF2ZSB0byBkbyBpdCBtYW51YWxseT8NCg0KIyAzIGJsYW5rL05BIHB1YiB5ZWFycw0KcHFfZW1wdHk8LXBxX21ldGFjbGVhbltwcV9tZXRhY2xlYW4kYFB1YmxpY2F0aW9uIGRhdGVgID09ICIiIHwgaXMubmEocHFfbWV0YWNsZWFuJGBQdWJsaWNhdGlvbiBkYXRlYCksXQ0KbnJvdyhwcV9lbXB0eSkNCiMgMCBvZiB0aGVzZSBibGFuay9OQSBwdWIgeWVhcnMgaGF2ZSBibGFuay9OQSAnZnVsbCB0ZXh0cycgYXMgd2VsbA0KbnJvdyhwcV9lbXB0eVtwcV9lbXB0eSRgRnVsbCB0ZXh0YCA9PSAiIiB8IGlzLm5hKHBxX2VtcHR5JGBGdWxsIHRleHRgKSxdKQ0KYGBgDQoNCmBgYHtyfQ0KIyBpZiBwdWJsaWNhdGlvbiBkYXRlIGlzIGVtcHR5IG9yIE5BLCBmaWxsIHdpdGggZGF0ZSBmcm9tIHB1YmxpY2F0aW9uIGluZm8NCnBxX21ldGFjbGVhbiA8LSBwcV9tZXRhY2xlYW4gJT4lIA0KICBtdXRhdGUoYFB1YmxpY2F0aW9uIGRhdGVgID0gaWZlbHNlKGBQdWJsaWNhdGlvbiBkYXRlYCA9PSAiIiB8IGlzLm5hKGBQdWJsaWNhdGlvbiBkYXRlYCksIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHVubGlzdChzdHJzcGxpdChgUHVibGljYXRpb24gaW5mb2AsIHNwbGl0PSJcXF18OiIpKVsyXSwgYFB1YmxpY2F0aW9uIGRhdGVgKSkNCmBgYA0KDQojIHJlLWNoZWNrIGVtcHR5IHB1YmxpY2F0aW9uIGRhdGVzIChzaG91bGQgYmUgMCBub3cpDQpgYGB7cn0NCiMgSG93IGRvIHlvdSB3YW50IHRvIHRyZWF0IE5BPw0KIyBzdWJzZXQgdGhlIGRhdGEgYnkgdGhlIE5BcywgZXhwbG9yZSB0aGVtICYgdGhlaXIgb3JpZ2luDQojIGNhbiB3ZSBwcm9ncmFtbWF0aWNhbGx5IGZpeCBpdCwgb3IgZG8gd2UgaGF2ZSB0byBkbyBpdCBtYW51YWxseT8NCg0KIyAzIGJsYW5rL05BIHB1YiB5ZWFycw0KcHFfZW1wdHk8LXBxX21ldGFjbGVhbltwcV9tZXRhY2xlYW4kYFB1YmxpY2F0aW9uIGRhdGVgID09ICIiIHwgaXMubmEocHFfbWV0YWNsZWFuJGBQdWJsaWNhdGlvbiBkYXRlYCksXQ0KbnJvdyhwcV9lbXB0eSkNCiMgMCBvZiB0aGVzZSBibGFuay9OQSBwdWIgeWVhcnMgaGF2ZSBibGFuay9OQSAnZnVsbCB0ZXh0cycgYXMgd2VsbA0KbnJvdyhwcV9lbXB0eVtwcV9lbXB0eSRgRnVsbCB0ZXh0YCA9PSAiIiB8IGlzLm5hKHBxX2VtcHR5JGBGdWxsIHRleHRgKSxdKQ0KYGBgDQoNCiMgbW9yZSBjbGVhbmluZ3MNCmBgYHtyfQ0KUHViVGl0bGVzPC0iQmFuZ29yIERhaWx5IE5ld3N8S2VubmViZWMgSm91cm5hbHxNYWluZSBUaW1lc3xNb3JuaW5nIFNlbnRpbmVsfFBvcnRsYW5kIFByZXNzIEhlcmFsZHxTdW4gSm91cm5hbCIgDQpUeHRGb3JtYXR0aW5nPC0iXGF8XGJ8XGZ8XG58XHJ8XHR8XHZ8XFxbfFxcXSINCg0KcHFfbWV0YWNsZWFuPC0gcHFfbWV0YWNsZWFuICU+JQ0KICBtdXRhdGUoYEZ1bGwgdGV4dGAgPSB0ZXh0Y2xlYW46OnJlcGxhY2VfaHRtbChgRnVsbCB0ZXh0YCwgc3ltYm9sPVRSVUUpKSAlPiUgIyByZXBsYWNlIGh0bWwsIHVybCwgc3ltYm9sLCB3aGl0ZSBmcm9tIHRleHRjbGVhbg0KICBtdXRhdGUoYEZ1bGwgdGV4dGAgPSB0ZXh0Y2xlYW46OnJlcGxhY2VfdXJsKGBGdWxsIHRleHRgLCByZXBsYWNlbWVudCA9ICc8PFVSTD4+JykpICU+JQ0KICBtdXRhdGUoYEZ1bGwgdGV4dGAgPSBnc3ViKFB1YlRpdGxlcywgIiAiLCBgRnVsbCB0ZXh0YCkpICU+JSAjIHJlbW92ZSBwdWJsaWNhdGlvbiB0aXRsZXMNCiAgbXV0YXRlKGBGdWxsIHRleHRgID0gZ3N1YihUeHRGb3JtYXR0aW5nLCAiICIsIGBGdWxsIHRleHRgKSkgJT4lICMgcmVtb3ZlIHRleHQgZm9ybWF0dGluZw0KICBtdXRhdGUoYEZ1bGwgdGV4dGAgPSBnc3ViKCJcIiIsICIgIiwgYEZ1bGwgdGV4dGApKSAlPiUgIyByZW1vdmUgZG91YmxlIHF1b3RlcyAiIg0KICBtdXRhdGUoYEZ1bGwgdGV4dGAgPSBzdHJpbmdpOjpzdHJpX3RyYW5zX2dlbmVyYWwoc3RyID0gYEZ1bGwgdGV4dGAsIGlkID0gIkxhdGluLUFTQ0lJIikpICU+JSAjIHJlbW92ZSBhY2NlbnRlZCBsZXR0ZXJzDQogIG11dGF0ZShgRnVsbCB0ZXh0YCA9IGN1c3RvbV9yZXBsYWNlX3N5bWJvbChgRnVsbCB0ZXh0YCkpICU+JSAjIHJlcGxhY2UgIiYiIHdpdGggImFuZCIgYW5kICIlIiB3aXRoICJwZXJjZW50Ig0KICBtdXRhdGUoYEZ1bGwgdGV4dGAgPSB0ZXh0Y2xlYW46OnJlcGxhY2Vfd2hpdGUoYEZ1bGwgdGV4dGApKSAjIHJlbW92ZSBleHRyYSB3aGl0ZSBzcGFjZSANCmBgYA0KDQojIElmIGR1cGxpY2F0ZSBmdWxsIHRleHQgZXhpc3RzLCBzZWxlY3QgZmlyc3QgaW4gc2VyaWVzIGFuZCBkdW1wIHRoZSByZXN0PyAtLSB5ZXMsIHNpbmNlIA0KIyBpbml0aWFsIGNsZWFuaW5nIGlzIHRvIHByb2R1Y2UgcmVwcmVzZW50YXRpdmUgY2F0ZWdvcmllcw0KDQpUaGVzZSBkb2N1bWVudCBJRHMgaGF2ZSByYW5kb20gc3BhY2VzIGluIHRoZSBwdWJsaWNhdGlvbiBkYXRlOyBpLmUuLCBNYXkgMDEsIDIwIDE0DQpESUQxPC0iMTU0MDc5MTkyNiINCkRJRDI8LSIyMjU3MTMzNDE1Ig0KDQpwcV9tZXRhZGF0YVtwcV9tZXRhZGF0YSRgUHJvUXVlc3QgZG9jdW1lbnQgSURgPT1ESUQxLF0NCg0KYGBge3J9DQojDQpoZWFkKHBxX21ldGFjbGVhbikNCnNvcnQocHFfbWV0YWNsZWFuJGBQdWJsaWNhdGlvbiBkYXRlYCkNCm51bVByZURlRHVwPC1ucm93KHBxX21ldGFjbGVhbikgIzY0NjENCnByaW50KHBhc3RlKCJOdW1iZXIgb2YgYXJ0aWNsZXMgcHJlLWRlZHVwOiIsbnVtUHJlRGVEdXApKQ0KbnVtRHVwPC1zdW0odGFibGUocHFfbWV0YWNsZWFuJGBGdWxsIHRleHRgKS0xKSAjIFRvdGFsIG51bWJlciBvZiBkdXBsaWNhdGUgZnVsbCB0ZXh0czogMTg0DQpwcmludChwYXN0ZSgiTnVtYmVyIG9mIGR1cGxpY2F0ZXM6IixudW1EdXApKQ0KDQojIGR1cGxpY2F0ZSBmbGFnIGlzIG5vdCBzdWZmaWNpZW50IHRvIHJlbW92ZSBhbGwgZHVwbGljYXRlcyAtIDUzIHJvd3MNCnBxX21ldGFkYXRhICU+JQ0KICBzZWxlY3QoYEZ1bGwgdGV4dGAsIGBQdWJsaWNhdGlvbiBpbmZvYCwgYFB1YmxpY2F0aW9uIGRhdGVgKSAlPiUNCiAgbXV0YXRlKGBQdWJsaWNhdGlvbiBkYXRlYCA9IGFzLkRhdGUoYFB1YmxpY2F0aW9uIGRhdGVgLCBmb3JtYXQgPSAiJWIgJWQsICVZIikpICU+JQ0KICBhcnJhbmdlKGBQdWJsaWNhdGlvbiBkYXRlYCkgJT4lDQogIGZpbHRlcihncmVwbCgnRHVwbGljYXRlJyxgUHVibGljYXRpb24gaW5mb2ApKQ0KDQpwcV9tZXRhY2xlYW4gPC0gcHFfbWV0YWNsZWFuICU+JQ0KICBmaWx0ZXIoIWdyZXBsKCdEdXBsaWNhdGUnLGBQdWJsaWNhdGlvbiBpbmZvYCkpICU+JQ0KICBncm91cF9ieShgRnVsbCB0ZXh0YCkgJT4lIA0KICBtdXRhdGUoYFB1YmxpY2F0aW9uIGRhdGVgID0gYXMuRGF0ZShgUHVibGljYXRpb24gZGF0ZWAsIGZvcm1hdCA9ICIlYiAlZCwgJVkiKSkgJT4lDQogIGFycmFuZ2UoYFB1YmxpY2F0aW9uIGRhdGVgKSAlPiUNCiAgc2xpY2UoMUwpDQoNCnBxX21ldGFjbGVhbiAlPiUNCiAgc2VsZWN0KGBQdWJsaWNhdGlvbiBpbmZvYCxgUHVibGljYXRpb24gZGF0ZWApDQoNCm51bVBvc3REZUR1cDwtbnJvdyhwcV9tZXRhY2xlYW4pDQpwcmludChwYXN0ZSgiTnVtYmVyIG9mIGFydGljbGVzIHBvc3QtZGVkdXA6IixudW1Qb3N0RGVEdXApKSAjIDYyNzcgdG8gNjIzOQ0KbnVtUmVtb3ZlZCA8LSAobnVtUHJlRGVEdXAgLSBudW1Qb3N0RGVEdXApDQpwcmludChwYXN0ZSgiTnVtYmVyIG9mIGR1cGxpY2F0ZXMgcmVtb3ZlZDoiLG51bVJlbW92ZWQpKSAjIDE4NCB0byAyMjINCmBgYA0KDQojIFNhdmUgdG8gcHFfbWV0YWNsZWFuLmNzdiBpZiBpdCBkb2Vzbid0IGFscmVhZHkgZXhpc3Qgb3Igb3ZlcndyaXRlID0gVFJVRQ0KYGBge3J9DQojIFNhdmUgY2xlYW5lZCBjb3JwdXMgdG8gbmV3IGZpbGUgDQpvdXRwdXRGaWxlTmFtZUNsZWFuIDwtIHBhc3RlMChyRmlsZU51bSwiX3BxX21ldGFjbGVhbiIpDQpvdXRwdXRGb2xkZXIgPSAiRGF0YS8wMl9Xb3JraW5nLyINCm91dHB1dEZpbGVDbGVhbiA9IHBhc3RlKG91dHB1dEZvbGRlcixvdXRwdXRGaWxlTmFtZUNsZWFuLCIuY3N2IixzZXA9IiIpDQoNCmlmICghZmlsZS5leGlzdHMob3V0cHV0RmlsZUNsZWFuKSB8IG92ZXJ3cml0ZSkgew0KICB3cml0ZS5jc3YocHFfbWV0YWNsZWFuLCBvdXRwdXRGaWxlQ2xlYW4sIHJvdy5uYW1lcz1GQUxTRSkNCn0NCmBgYA==